{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Question Answering LLM Fine-tuning"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Todo: ensure these are in requirements.txt and version compatability"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "id": "SoOBU-u1qT8F",
    "tags": []
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore') #Some operations warn inside a loop"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "tsG7Vz8wBPhf"
   },
   "source": [
    "## Listing 14.8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_processor_type():\n",
    "    gpu_device = torch.device(\"cuda:0\")\n",
    "    cpu_device = torch.device(\"cpu\")\n",
    "    return gpu_device or cpu_device\n",
    "\n",
    "def get_processor_device():\n",
    "    return 0 if torch.cuda.is_available() else -1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "adpnkDdJBjkr",
    "outputId": "3be526cf-2326-4abd-a96f-5527b423b3c9",
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processor: cuda:0\n",
      "Device id: -1\n"
     ]
    }
   ],
   "source": [
    "print(\"Processor: \" + str(get_processor_type()))\n",
    "print(\"Device id: \" + str(get_processor_device()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Add Google Collab integration back"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Grab a pre-generated copy of the golden set in case you skipped training it in Listing 14.7"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cloning into 'question-answering'...\n",
      "remote: Enumerating objects: 16, done.\u001b[K\n",
      "remote: Counting objects: 100% (16/16), done.\u001b[K\n",
      "remote: Compressing objects: 100% (12/12), done.\u001b[K\n",
      "remote: Total 16 (delta 2), reused 14 (delta 2), pack-reused 0 (from 0)\u001b[K\n",
      "Receiving objects: 100% (16/16), 92.27 KiB | 1.84 MiB/s, done.\n",
      "Resolving deltas: 100% (2/2), done.\n",
      "Already up to date.\n"
     ]
    }
   ],
   "source": [
    "!cd data && [ ! -d \"question-answering\" ] && git clone --depth=1 https://github.com/ai-powered-search/question-answering\n",
    "!cd data && [ -d \"question-answering\" ] && cd question-answering && git pull "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "SD8RnJGdDRmI",
    "outputId": "5a008fe6-3e14-4b11-8ddf-8b938a374306",
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "83517d6adb97453ca121458b65472e62",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "tokenizer_config.json:   0%|          | 0.00/25.0 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8fd32b5665a1452286b41d9120916eaf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "vocab.json:   0%|          | 0.00/899k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "18069c80116f489f9f5835233a5a37aa",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "merges.txt:   0%|          | 0.00/456k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "73c16403aa16417fb5e4412ff22bb005",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "tokenizer.json:   0%|          | 0.00/1.36M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d4dabce2855a4b13b92bbd0d32ab6efc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "config.json:   0%|          | 0.00/481 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "RobertaTokenizerFast(name_or_path='roberta-base', vocab_size=50265, model_max_length=512, is_fast=True, padding_side='right', truncation_side='right', special_tokens={'bos_token': '<s>', 'eos_token': '</s>', 'unk_token': '<unk>', 'sep_token': '</s>', 'pad_token': '<pad>', 'cls_token': '<s>', 'mask_token': '<mask>'}, clean_up_tokenization_spaces=False),  added_tokens_decoder={\n",
       "\t0: AddedToken(\"<s>\", rstrip=False, lstrip=False, single_word=False, normalized=True, special=True),\n",
       "\t1: AddedToken(\"<pad>\", rstrip=False, lstrip=False, single_word=False, normalized=True, special=True),\n",
       "\t2: AddedToken(\"</s>\", rstrip=False, lstrip=False, single_word=False, normalized=True, special=True),\n",
       "\t3: AddedToken(\"<unk>\", rstrip=False, lstrip=False, single_word=False, normalized=True, special=True),\n",
       "\t50264: AddedToken(\"<mask>\", rstrip=False, lstrip=True, single_word=False, normalized=False, special=True),\n",
       "}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import transformers\n",
    "tokenizer = transformers.RobertaTokenizerFast.from_pretrained('roberta-base')\n",
    "assert isinstance(tokenizer, transformers.PreTrainedTokenizerFast)\n",
    "tokenizer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "AOWOiSLIDSSG"
   },
   "source": [
    "## Listing 14.9"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "BYGeRBSCByyd"
   },
   "source": [
    "### Hyperparameter alert!\n",
    "\n",
    "Hyperparameters are serious business.  Memory and Computation resources are very very finite.  We do our best to limit visible scope, both for the model and for the speed.  We also need to do this since the tensors we use during training and evaluation must have a fixed shape.  This shape must be the same for all examples we provide to the trainer and evaluator.\n",
    "\n",
    "We accomplish this with a window sliding technique and by right-padding.  Windowing and padding will make sure everything is the same shape."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 269,
     "referenced_widgets": [
      "23792304678646fead3907da4165e57b",
      "969b3918dc364edb91dbb6728f746a57",
      "e721d450acb04c97a3e8de1073923c77",
      "1cee1c9da6ab4ef482ccf85c9e3836fc",
      "c7a4f2a594e146ecb3e0ca70c4aba77c",
      "1ad95d2c2a1d456ca2bee9b7506b57e4",
      "80f5f671101249bca168c056d7c1b8ec",
      "81c90ba9eb0342bd8e7c09d944b5ae81",
      "2a7d2eecba714e9987b8cd8a7dfe905c",
      "a39fa854068a488683a09bfa29d881e6",
      "5a7db3fbce2b437581f4159e71cdc501",
      "da51c5e2afaa453bb5b71a2ae7ae16a1",
      "0e2452f8efea42de86f1c05232c27aad",
      "4d9d84193fe346e9b0eab69cc34e4cce",
      "7fca011072fd4f33bab70d0353b7186d",
      "1f8f290522874db49bd50e74b708ccf2",
      "b6c492f6e031492c848b6f1f17314f42",
      "6e67e54ae28149ac84404810b68204f9",
      "183ddcb59b9f4fc0b29f5ec64a4fcfa7",
      "30543df8298249d08016d19bb73d8c0a",
      "570ba43586a34d219d26877d3dfe6c72",
      "a6b798a73b5d414e947f3940ea8f9b2d",
      "c4daf132dbcd4b6cb0c59bfe7ca44c25",
      "be867889e9464e67a64e0153706e62fb"
     ]
    },
    "id": "x5sKA3S-oLhE",
    "outputId": "0ed8a717-8057-4afd-f3a1-e2a06f23db90",
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "802f05ecfd8544ae8d2787a6c77ad4a7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/125 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Encoding(num_tokens=384, attributes=[ids, type_ids, tokens, offsets, attention_mask, special_tokens_mask, overflowing])\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d9a1900fcc584becb9d5a1663bc6b37e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/32 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Encoding(num_tokens=384, attributes=[ids, type_ids, tokens, offsets, attention_mask, special_tokens_mask, overflowing])\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3f6b804f27d743309c6e0bde4858889c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/10 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Encoding(num_tokens=384, attributes=[ids, type_ids, tokens, offsets, attention_mask, special_tokens_mask, overflowing])\n"
     ]
    }
   ],
   "source": [
    "#This method adopted from the following example notebook:\n",
    "#https://github.com/huggingface/notebooks/blob/master/examples/question_answering.ipynb\n",
    "#Copyright 2021, Huggingface.  Apache 2.0 license.\n",
    "import datasets\n",
    "\n",
    "file = \"data/question-answering/question-answering-training-set\"\n",
    "datadict = datasets.load_from_disk(file)\n",
    "\n",
    "def tokenize_dataset(examples):\n",
    "\n",
    "    maximum_tokens = 384 # This will be the number of tokens in BOTH the question and context\n",
    "    document_overlap = 128 # Sometimes we need to split the context into smaller chunks, so we will overlap with this window\n",
    "    pad_on_right = tokenizer.padding_side == \"right\"\n",
    "    \n",
    "    # Tokenize our examples with truncation and padding, but keep the overflows using a stride. This results\n",
    "    # in one example possible giving several features when a context is long, each of those features having a\n",
    "    # context that overlaps a bit the context of the previous feature.\n",
    "    tokenized_examples = tokenizer(\n",
    "        examples[\"question\" if pad_on_right else \"context\"],\n",
    "        examples[\"context\" if pad_on_right else \"question\"],\n",
    "        truncation=\"only_second\" if pad_on_right else \"only_first\",\n",
    "        max_length=maximum_tokens,\n",
    "        stride=document_overlap,\n",
    "        return_overflowing_tokens=True,\n",
    "        return_offsets_mapping=True,\n",
    "        padding=\"max_length\"\n",
    "    )\n",
    "    \n",
    "    print(tokenized_examples[0])\n",
    "\n",
    "    # Since one example might give us several features if it has a long context, we need a map from a feature to\n",
    "    # its corresponding example. This key gives us just that.\n",
    "    sample_mapping = tokenized_examples.pop(\"overflow_to_sample_mapping\")\n",
    "    # The offset mappings will give us a map from token to character position in the original context. This will\n",
    "    # help us compute the start_positions and end_positions.\n",
    "    offset_mapping = tokenized_examples.pop(\"offset_mapping\")\n",
    "\n",
    "    # Let's label those examples!\n",
    "    tokenized_examples[\"start_positions\"] = []\n",
    "    tokenized_examples[\"end_positions\"] = []\n",
    "\n",
    "    for i, offsets in enumerate(offset_mapping):\n",
    "        # We will label impossible answers with the index of the CLS token.\n",
    "        input_ids = tokenized_examples[\"input_ids\"][i]\n",
    "        cls_index = input_ids.index(tokenizer.cls_token_id)\n",
    "\n",
    "        # Grab the sequence corresponding to that example (to know what is the context and what is the question).\n",
    "        sequence_ids = tokenized_examples.sequence_ids(i)\n",
    "\n",
    "        # One example can give several spans, this is the index of the example containing this span of text.\n",
    "        sample_index = sample_mapping[i]\n",
    "        answers = examples[\"answers\"][sample_index]\n",
    "        # If no answers are given, set the cls_index as answer.\n",
    "        if len(answers[\"answer_start\"]) == 0:\n",
    "            tokenized_examples[\"start_positions\"].append(cls_index)\n",
    "            tokenized_examples[\"end_positions\"].append(cls_index)\n",
    "        else:\n",
    "            # Start/end character index of the answer in the text.\n",
    "            start_char = answers[\"answer_start\"][0]\n",
    "            end_char = start_char + len(answers[\"text\"][0])\n",
    "\n",
    "            # Start token index of the current span in the text.\n",
    "            token_start_index = 0\n",
    "            while sequence_ids[token_start_index] != (1 if pad_on_right else 0):\n",
    "                token_start_index += 1\n",
    "\n",
    "            # End token index of the current span in the text.\n",
    "            token_end_index = len(input_ids) - 1\n",
    "            while sequence_ids[token_end_index] != (1 if pad_on_right else 0):\n",
    "                token_end_index -= 1\n",
    "\n",
    "            # Detect if the answer is out of the span (in which case this feature is labeled with the CLS index).\n",
    "            if not (offsets[token_start_index][0] <= start_char and offsets[token_end_index][1] >= end_char):\n",
    "                tokenized_examples[\"start_positions\"].append(cls_index)\n",
    "                tokenized_examples[\"end_positions\"].append(cls_index)\n",
    "            else:\n",
    "                # Otherwise move the token_start_index and token_end_index to the two ends of the answer.\n",
    "                # Note: we could go after the last offset if the answer is the last word (edge case).\n",
    "                while token_start_index < len(offsets) and offsets[token_start_index][0] <= start_char:\n",
    "                    token_start_index += 1\n",
    "                tokenized_examples[\"start_positions\"].append(token_start_index - 1)\n",
    "                while offsets[token_end_index][1] >= end_char:\n",
    "                    token_end_index -= 1\n",
    "                tokenized_examples[\"end_positions\"].append(token_end_index + 1)\n",
    "\n",
    "    return tokenized_examples\n",
    "\"\"\"\n",
    "To apply this function on all the sentences (or pairs of sentences) in our dataset, we just use the map method of our dataset object we created earlier. \n",
    "This will apply the function on all the elements of all the splits in dataset, so our training, validation and testing data will be preprocessed in one single command. \n",
    "Since our preprocessing changes the number of samples, we need to remove the old columns when applying it.\n",
    " --Huggingface\n",
    "\"\"\"\n",
    "tokenized_datasets = datadict.map(tokenize_dataset, batched=True, remove_columns=datadict[\"train\"].column_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "id": "6_f9znBQoLmx",
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4d88fab70625493195f7adab2b04a6fa",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Saving the dataset (0/1 shards):   0%|          | 0/156 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "771cd059a6ce458cb0da690017178ef7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Saving the dataset (0/1 shards):   0%|          | 0/44 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d68b1f6f03174fb38b601c5af1aeec37",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Saving the dataset (0/1 shards):   0%|          | 0/15 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "tokenized_datasets.save_to_disk(\"data/question-answering/qa-training-set-tokenized\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "hAVKmdsLESX1"
   },
   "source": [
    "## Listing 14.10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 116,
     "referenced_widgets": [
      "f0866a7cfe204095afad09de79905a0a",
      "7f71c1720faa457c86d865d90592eb56",
      "4348587b803e4d70959b0f75d57c8ef8",
      "acafe70967d045ee9094b1d94baac658",
      "60bd3b12bf8748d19a55ccba0314a9ef",
      "75a659744d0a4bbc84755807206f2650",
      "a2863a28644b446faaf46a7a2172875b",
      "1b77a9ff0ccc4b458e78900f35dec862",
      "fca3e362da2b40f6924c582bf0447a9b",
      "437a30a370a84d76bc3aa94407bc4787",
      "88c198332bda4d14a00ab377dafabe60",
      "ce9ce5ba37a247eda5061c5a0e7e1aed",
      "b9aa0f93a5b049888df95bd48eae4a01",
      "498e555d731245d29d4ebc36b51772d3",
      "784cbfb0bd134466a1b9d7de2aadd155",
      "9eea5e57914342bdb7b8c53df0e95516"
     ]
    },
    "id": "7_I6tE0NqT8S",
    "outputId": "74fb670e-37bb-44cc-c858-06928b3c1a4f",
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5979bb1b726540c797c4133477400d25",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "config.json:   0%|          | 0.00/571 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3c12c9e41dfc42d69f59ce920d9ac6c2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model.safetensors:   0%|          | 0.00/496M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from transformers import RobertaForQuestionAnswering, TrainingArguments, Trainer, default_data_collator\n",
    "\n",
    "model = RobertaForQuestionAnswering.from_pretrained('deepset/roberta-base-squad2')\n",
    "\n",
    "training_args = TrainingArguments(\n",
    "    evaluation_strategy=\"epoch\",                          # evaluate loss per epoch\n",
    "    num_train_epochs=3,                                   # total # of training epochs\n",
    "    per_device_train_batch_size=16,                       # batch size per device during training\n",
    "    per_device_eval_batch_size=64,                        # batch size for evaluation\n",
    "    warmup_steps=500,                                     # number of warmup steps for learning rate scheduler\n",
    "    weight_decay=0.01,                                    # strength of weight decay\n",
    "    logging_dir=\"data/question-answering/logs\",     # directory for storing logs\n",
    "    output_dir=\"data/question-answering/results\")   # output directory\n",
    "\n",
    "trainer = Trainer(\n",
    "    model=model,                                          # the instantiated 🤗 Transformers model to be trained\n",
    "    args=training_args,                                   # training arguments, defined above\n",
    "    data_collator=default_data_collator,                  \n",
    "    tokenizer=tokenizer,                                  \n",
    "    train_dataset=tokenized_datasets[\"train\"],            # training dataset\n",
    "    eval_dataset=tokenized_datasets[\"test\"])              # evaluation dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "uI306rN2EsXC"
   },
   "source": [
    "## Listing 14.11"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 207
    },
    "id": "op6PAcrEtShU",
    "outputId": "bb2db305-fe98-4355-a4a7-32240ae0e487",
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='30' max='30' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [30/30 06:11, Epoch 3/3]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>No log</td>\n",
       "      <td>2.177583</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>No log</td>\n",
       "      <td>2.016354</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>No log</td>\n",
       "      <td>1.923386</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "TrainOutput(global_step=30, training_loss=2.5694539388020834, metrics={'train_runtime': 383.9234, 'train_samples_per_second': 1.219, 'train_steps_per_second': 0.078, 'total_flos': 91715161614336.0, 'train_loss': 2.5694539388020834, 'epoch': 3.0})"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "id": "HipTWZhOtamC",
    "tags": []
   },
   "outputs": [],
   "source": [
    "model_name = \"data/question-answering/roberta-base-squad2-fine-tuned\"\n",
    "trainer.save_model(model_name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "5vxV-ulcF04F"
   },
   "source": [
    "## Listing 14.12"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 106
    },
    "id": "pwbKz-dTtqY1",
    "outputId": "be6c055f-cca2-4b41-a144-ccb03d262ddb",
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='1' max='1' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [1/1 : < :]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "{'eval_loss': 1.7502323389053345,\n",
       " 'eval_runtime': 3.2996,\n",
       " 'eval_samples_per_second': 4.546,\n",
       " 'eval_steps_per_second': 0.303,\n",
       " 'epoch': 3.0}"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "evaluation = trainer.evaluate(eval_dataset=tokenized_datasets[\"validation\"])\n",
    "display(evaluation)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "C7bMVXrsJvsV"
   },
   "source": [
    "## Listing 14.13"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "Gd66wSCZJv1Q",
    "outputId": "349b2aeb-7954-4f80-fad2-3eaa35ee689a",
    "tags": []
   },
   "outputs": [],
   "source": [
    "from transformers import pipeline\n",
    "\n",
    "device = get_processor_device()\n",
    "model_name = \"data/question-answering/roberta-base-squad2-fine-tuned\"\n",
    "nlp2 = pipeline(\"question-answering\", model=model_name, tokenizer=model_name,\n",
    "                device=device)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "T_iwwK7LP2ZL"
   },
   "source": [
    "## Listing 14.14"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "id": "4EvM2U_Vt7Eg",
    "tags": []
   },
   "outputs": [],
   "source": [
    "def answer_questions(examples):\n",
    "    answers = []\n",
    "    success = 0\n",
    "    for example in examples:\n",
    "        question = {\"question\": example[\"question\"][0],\n",
    "                    \"context\": example[\"context\"][0]}\n",
    "        answer = nlp2(question)\n",
    "        label = example[\"answers\"][0][\"text\"][0]\n",
    "        result = answer[\"answer\"]\n",
    "        print(question[\"question\"])\n",
    "        print(\"Label:\", label)\n",
    "        print(\"Result:\", result)\n",
    "        print(\"----------\")\n",
    "        success += (1 if (label == result) else 0)\n",
    "        answers.append(answer)\n",
    "    print(f\"{success}/{len(examples)} correct\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "LlDZJ857uM02",
    "outputId": "f3515329-1662-452d-84a9-2aaa18490286",
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "How to get pine sap off my teeth\n",
      "Label: Take a small amount of margarine and rub on the sap\n",
      "Result: Take a small amount of margarine and rub on the sap\n",
      "----------\n",
      "Why are backpack waist straps so long?\n",
      "Label: The most backpacks have only one size for everyone\n",
      "Result: The most backpacks have only one size for everyone\n",
      "----------\n",
      "What can I do to prevent altitude sickness?\n",
      "Label: acclimate\n",
      "Result: acclimate\n",
      "----------\n",
      "What group of people call themselves \"Outdoor Influencers\", and what do they do regarding natural areas of land?\n",
      "Label: raise awareness for important causes to protect these lands\n",
      "Result: raise awareness for important causes to protect these lands\n",
      "----------\n",
      "When to sharpen crampons?\n",
      "Label: when I am expecting icy conditions\n",
      "Result: when I am expecting icy conditions\n",
      "----------\n",
      "What is the benefit to telemark skiing?\n",
      "Label: allow skiers to skin up back-country slopes with a more natural and efficient stride\n",
      "Result: more natural and efficient stride\n",
      "----------\n",
      "What do you do for sun allergy?\n",
      "Label: cover up, with clothes and sun screen\n",
      "Result: cover up, with clothes and sun screen\n",
      "----------\n",
      "What happens when someone dies on Mount Everest?\n",
      "Label: they are left there\n",
      "Result: they die in the attempt\n",
      "----------\n",
      "What are good locations to find wild strawberries?\n",
      "Label: along the edges of abandoned farm fields\n",
      "Result: along the edges of abandoned farm fields\n",
      "----------\n",
      "How efficient is the Altai skis \"the Hok\"?\n",
      "Label: you can easily glide in one direction (forward) and if you try to glide backwards, the fur will \"bristle up\"\n",
      "Result: you can easily go uphill, without (much) affecting forward gliding performance\n",
      "----------\n",
      "7/10 correct\n"
     ]
    }
   ],
   "source": [
    "datadict[\"validation\"].set_format(type=\"pandas\", output_all_columns=True)\n",
    "validation_examples = [example for example in datadict[\"validation\"]]\n",
    "answer_questions(validation_examples)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "YojbeHCAuph-",
    "tags": []
   },
   "outputs": [],
   "source": [
    "#This is an illustration of grid search.  For the Transformers builtin, see https://huggingface.co/transformers/main_classes/trainer.html#transformers.Trainer.hyperparameter_search\n",
    "\n",
    "from transformers import RobertaForQuestionAnswering, TrainingArguments, Trainer, default_data_collator\n",
    "import torch\n",
    "\n",
    "def grid_search_finetuning(tokenized_datasets):\n",
    "    epochs = [4]\n",
    "    batches = [16, 18]\n",
    "    warmups = [50, 250, 500]\n",
    "  \n",
    "    for epoch in epochs:\n",
    "        for batch in batches:\n",
    "            for warmup in warmups:\n",
    "                model = RobertaForQuestionAnswering.from_pretrained(\"deepset/roberta-base-squad2\")\n",
    "                name = \"_\".join([\"epochs\", str(epoch), \"batchsize\", str(batch), \"warmup\", str(warmup)])\n",
    "\n",
    "                print(\"-----------------------------------------------\\n\")\n",
    "                print(name)\n",
    "                training_args = TrainingArguments(\n",
    "                    evaluation_strategy = \"epoch\",                         # evaluate loss per epoch\n",
    "                    num_train_epochs=epoch,                                # total # of training epochs\n",
    "                    per_device_train_batch_size=batch,                     # batch size per device during training\n",
    "                    per_device_eval_batch_size=64,                         # batch size for evaluation\n",
    "                    warmup_steps=warmup,                                   # number of warmup steps for learning rate scheduler\n",
    "                    weight_decay=0.01,                                     # strength of weight decay\n",
    "                    logging_dir=\"data/question-answering/logs_\" + name,  # directory for storing logs\n",
    "                    output_dir=\"data/question-answering/results_\" + name # output directory\n",
    "                )\n",
    "\n",
    "                trainer = Trainer(\n",
    "                    model=model,                                          # the instantiated 🤗 Transformers model to be trained\n",
    "                    args=training_args,                                   # training arguments, defined above\n",
    "                    data_collator=default_data_collator,                  \n",
    "                    tokenizer=tokenizer,                                  \n",
    "                    train_dataset=tokenized_datasets[\"train\"],            # training dataset\n",
    "                    eval_dataset=tokenized_datasets[\"test\"]               # evaluation dataset\n",
    "                )\n",
    "\n",
    "                training_outputs = trainer.train()\n",
    "                print(\"\\nTraining Loss:\", training_outputs.training_loss)\n",
    "                evaluation_outputs = trainer.evaluate(eval_dataset=tokenized_datasets[\"validation\"])\n",
    "                print(\"Evaluation Loss:\", evaluation_outputs[\"eval_loss\"])\n",
    "                print(training_outputs)\n",
    "                print(evaluation_outputs)\n",
    "\n",
    "                del trainer\n",
    "                del model\n",
    "\n",
    "grid_search_finetuning(tokenized_datasets)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Up next: [Question Answering demo application](4.question-answering-demo-application.ipynb)"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [
    "tsG7Vz8wBPhf",
    "ffcQIkCDBbP3",
    "AOWOiSLIDSSG",
    "BYGeRBSCByyd",
    "hAVKmdsLESX1",
    "uI306rN2EsXC"
   ],
   "name": "5.ch13-question-answering-GPU-fine-tuning.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "0e2452f8efea42de86f1c05232c27aad": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "183ddcb59b9f4fc0b29f5ec64a4fcfa7": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_a6b798a73b5d414e947f3940ea8f9b2d",
      "max": 1,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_570ba43586a34d219d26877d3dfe6c72",
      "value": 1
     }
    },
    "1ad95d2c2a1d456ca2bee9b7506b57e4": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "1b77a9ff0ccc4b458e78900f35dec862": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "1cee1c9da6ab4ef482ccf85c9e3836fc": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_81c90ba9eb0342bd8e7c09d944b5ae81",
      "placeholder": "​",
      "style": "IPY_MODEL_80f5f671101249bca168c056d7c1b8ec",
      "value": " 1/1 [01:41&lt;00:00, 101.85s/ba]"
     }
    },
    "1f8f290522874db49bd50e74b708ccf2": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "23792304678646fead3907da4165e57b": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_e721d450acb04c97a3e8de1073923c77",
       "IPY_MODEL_1cee1c9da6ab4ef482ccf85c9e3836fc"
      ],
      "layout": "IPY_MODEL_969b3918dc364edb91dbb6728f746a57"
     }
    },
    "2a7d2eecba714e9987b8cd8a7dfe905c": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_5a7db3fbce2b437581f4159e71cdc501",
       "IPY_MODEL_da51c5e2afaa453bb5b71a2ae7ae16a1"
      ],
      "layout": "IPY_MODEL_a39fa854068a488683a09bfa29d881e6"
     }
    },
    "30543df8298249d08016d19bb73d8c0a": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_be867889e9464e67a64e0153706e62fb",
      "placeholder": "​",
      "style": "IPY_MODEL_c4daf132dbcd4b6cb0c59bfe7ca44c25",
      "value": " 1/1 [01:41&lt;00:00, 101.52s/ba]"
     }
    },
    "4348587b803e4d70959b0f75d57c8ef8": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "Downloading: 100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_75a659744d0a4bbc84755807206f2650",
      "max": 571,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_60bd3b12bf8748d19a55ccba0314a9ef",
      "value": 571
     }
    },
    "437a30a370a84d76bc3aa94407bc4787": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "498e555d731245d29d4ebc36b51772d3": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "4d9d84193fe346e9b0eab69cc34e4cce": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "570ba43586a34d219d26877d3dfe6c72": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "5a7db3fbce2b437581f4159e71cdc501": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_4d9d84193fe346e9b0eab69cc34e4cce",
      "max": 1,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_0e2452f8efea42de86f1c05232c27aad",
      "value": 1
     }
    },
    "60bd3b12bf8748d19a55ccba0314a9ef": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "6e67e54ae28149ac84404810b68204f9": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "75a659744d0a4bbc84755807206f2650": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "784cbfb0bd134466a1b9d7de2aadd155": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "7f71c1720faa457c86d865d90592eb56": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "7fca011072fd4f33bab70d0353b7186d": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "80f5f671101249bca168c056d7c1b8ec": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "81c90ba9eb0342bd8e7c09d944b5ae81": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "88c198332bda4d14a00ab377dafabe60": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "Downloading: 100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_498e555d731245d29d4ebc36b51772d3",
      "max": 496313727,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_b9aa0f93a5b049888df95bd48eae4a01",
      "value": 496313727
     }
    },
    "969b3918dc364edb91dbb6728f746a57": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "9eea5e57914342bdb7b8c53df0e95516": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "a2863a28644b446faaf46a7a2172875b": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "a39fa854068a488683a09bfa29d881e6": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "a6b798a73b5d414e947f3940ea8f9b2d": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "acafe70967d045ee9094b1d94baac658": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_1b77a9ff0ccc4b458e78900f35dec862",
      "placeholder": "​",
      "style": "IPY_MODEL_a2863a28644b446faaf46a7a2172875b",
      "value": " 571/571 [00:00&lt;00:00, 2.93kB/s]"
     }
    },
    "b6c492f6e031492c848b6f1f17314f42": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_183ddcb59b9f4fc0b29f5ec64a4fcfa7",
       "IPY_MODEL_30543df8298249d08016d19bb73d8c0a"
      ],
      "layout": "IPY_MODEL_6e67e54ae28149ac84404810b68204f9"
     }
    },
    "b9aa0f93a5b049888df95bd48eae4a01": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "be867889e9464e67a64e0153706e62fb": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "c4daf132dbcd4b6cb0c59bfe7ca44c25": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "c7a4f2a594e146ecb3e0ca70c4aba77c": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "ce9ce5ba37a247eda5061c5a0e7e1aed": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_9eea5e57914342bdb7b8c53df0e95516",
      "placeholder": "​",
      "style": "IPY_MODEL_784cbfb0bd134466a1b9d7de2aadd155",
      "value": " 496M/496M [02:48&lt;00:00, 2.94MB/s]"
     }
    },
    "da51c5e2afaa453bb5b71a2ae7ae16a1": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_1f8f290522874db49bd50e74b708ccf2",
      "placeholder": "​",
      "style": "IPY_MODEL_7fca011072fd4f33bab70d0353b7186d",
      "value": " 1/1 [01:41&lt;00:00, 101.65s/ba]"
     }
    },
    "e721d450acb04c97a3e8de1073923c77": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_1ad95d2c2a1d456ca2bee9b7506b57e4",
      "max": 1,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_c7a4f2a594e146ecb3e0ca70c4aba77c",
      "value": 1
     }
    },
    "f0866a7cfe204095afad09de79905a0a": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_4348587b803e4d70959b0f75d57c8ef8",
       "IPY_MODEL_acafe70967d045ee9094b1d94baac658"
      ],
      "layout": "IPY_MODEL_7f71c1720faa457c86d865d90592eb56"
     }
    },
    "fca3e362da2b40f6924c582bf0447a9b": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_88c198332bda4d14a00ab377dafabe60",
       "IPY_MODEL_ce9ce5ba37a247eda5061c5a0e7e1aed"
      ],
      "layout": "IPY_MODEL_437a30a370a84d76bc3aa94407bc4787"
     }
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
